package cn.doitedu.dwh

import java.io.{ByteArrayOutputStream, DataOutputStream}

import org.apache.spark.sql.SparkSession
import org.roaringbitmap.RoaringBitmap

/**
 * @author 涛哥
 * @nick_name "deep as the sea"
 * @contact qq:657270652 wx:doit_edu
 * @site www.doitedu.cn
 * @date 2021-06-12
 * @desc 流量多维报表分析，base cuboid计算
 */
object ApplogTrfcBaseCuboidCalc {
  def main(args: Array[String]): Unit = {

    val spark = SparkSession
      .builder()
      .config("spark.sql.shuffle.partitions","1")
      .appName("")
      .enableHiveSupport()
      .master("local")
      .getOrCreate()


    // 加载数据
    //val table = spark.read.table("test.bitmap_cnt_dstc")


    val arr2bm = (arr:Array[Int])=>{
      val bitmap: RoaringBitmap = RoaringBitmap.bitmapOf(arr: _*)

      // 将得到的bitmap序列化成字节数组
      val out = new ByteArrayOutputStream()
      val dataOutput = new DataOutputStream(out)
      bitmap.serialize(dataOutput)

      out.toByteArray
    }

    spark.udf.register("arr2bm",arr2bm)


    // 算pv、uv
    val res = spark.sql(
      """
        |
        |select
        |
        |   province,
        |   city,
        |   region,
        |   arr2bm(collect_set(guid)) as guid_bmp,  -- 类型其实就是一个字节数组（bitmap对象的序列化结果）
        |   count(1) as pv_cnt
        |
        |from test.bitmap_cnt_dstc
        |group by province,city,region
        |
        |
        |""".stripMargin)

    /**
     * +--------+------+------+-------------------------------------------------------------+------+
     * |province|city  |region|guid_bmp                                                     |pv_cnt|
     * +--------+------+------+-------------------------------------------------------------+------+
     * |江苏省  |南通市|下关区|[3A 30 00 00 01 00 00 00 00 00 01 00 10 00 00 00 01 00 02 00]|18    |
     * |江苏省  |南通市|白领区|[3A 30 00 00 01 00 00 00 00 00 01 00 10 00 00 00 02 00 03 00]|2     |
     * |江苏省  |南通市|富豪区|[3A 30 00 00 01 00 00 00 00 00 00 00 10 00 00 00 03 00]      |1     |
     * |江苏省  |苏州市|园林区|[3A 30 00 00 01 00 00 00 00 00 00 00 10 00 00 00 01 00]      |2     |
     * |江苏省  |苏州市|虎跳区|[3A 30 00 00 01 00 00 00 00 00 00 00 10 00 00 00 04 00]      |1     |
     * |河北省  |保定市|幸福区|[3A 30 00 00 01 00 00 00 00 00 01 00 10 00 00 00 01 00 02 00]|2     |
     * |河北省  |保定市|安乐区|[3A 30 00 00 01 00 00 00 00 00 01 00 10 00 00 00 01 00 03 00]|2     |
     * |河北省  |邯郸市|下沙区|[3A 30 00 00 01 00 00 00 00 00 01 00 10 00 00 00 02 00 04 00]|2     |
     * |河北省  |邯郸市|上水区|[3A 30 00 00 01 00 00 00 00 00 00 00 10 00 00 00 05 00]      |2     |
     * +--------+------+------+-------------------------------------------------------------+------+
     */


    res.write.saveAsTable("test.bitmap_cnt_dstc_base_cuboid")


    spark.close()

  }

}
